# -*- coding: utf-8 -*-
'''
Created on May 24, 2010
@author: Long Hoang Giang
'''
import datetime, time, os, re
import commonlib
import traceback
import workerpool
from MongoModel import MongoModel

MONGO_SERVER = '27.0.12.106'
MONGO_PORT = 27017
LOG = commonlib.getMongoLog(MONGO_SERVER, MONGO_PORT, 'wapvn_xoso')


class Task(workerpool.Job):
    
    def __init__(self, code, mien, fcallBack):
        self.code = code
        self.mien = mien
        self.fcallBack = fcallBack
    
    def run(self):
        self.fcallBack(self.code, self.mien)

class WapVN(MongoModel):
    
    __giai = ['ĐB', 'ĐB6số', 'G1', 'G2', 'G3', 'G4', 'G5', 'G6', 'G7', 'G8']
    __url = 'http://xoso.wap.vn/detail.jsp?code={0}'
    
    def __init__(self, host='localhost', port=27017):
        MongoModel.__init__(self, host, port)
    
    def standardizeTimeStr(self, timeStr):
        if commonlib.dataTypeName(timeStr) == 'str':
            preg = re.compile(r'(\d+)/(\d+)/(\d+)')
            m = preg.search(timeStr)
            if m != '':
                return m.group(0)
        return ''
            
    def processDetail(self, ma, region):
        LOG.debug('start processDetail(%s, %s)' % (ma, region))
        url = self.__url.format(ma)
        try:
            tree = commonlib.getXMLTree(url)
            name = commonlib.getElementText(tree.xpath("/html/body/div[4]"))
            date = self.standardizeTimeStr(commonlib.getElementText(tree.xpath("/html/body/div[5]")))
            dateobj = datetime.datetime.strptime(date, "%d/%m/%Y")
            result = {'_id': str(hash(ma + date)),
                'name': name,
                'ma': ma,
                'mien': region,
                'date': date,
                'dateobj': dateobj
                }
            for i in self.__giai:
                kq = commonlib.getElementText(tree.xpath("//td[contains(., '{0}')]/following-sibling::*[1]".format(i).decode('utf-8'))).strip()
                if i == 'ĐB': i = 'DB'
                elif i == 'ĐB6số': i = 'DB6so'
                result[i] = kq
            result['timestamp'] = time.time()
            result['lastupdate'] = datetime.datetime.now()
            db = self.connection['xoso']
            collection = db[ma]         # NOTE: luu giong database cu~
            collection.save(result)
            for i, v in result.items():
                print '{0}: {1}'.format(i, v)
        except:
            LOG.error(traceback.format_exc())
        LOG.debug('finished processDetail(%s, %s, %s)' % (ma, region, url))
            
    def process(self):
        LOG.debug('start process with 10 threads')
        mienBac = ['XSTD']
        mienTrung = ['XSBDH','XSDLK','XSGL','XSKH','XSKT','XSNT','XSPY','XSQB','XSQNM','XSQNI','XSQT','XSTTH','XSDNG','XSDNO']
        mienNam = ['XSAG','XSBD','XSBP','XSBTH','XSBL','XSBTR','XSCM','XSCT','XSHG','XSHCM','XSKG','XSLA','XSST','XSTG','XSTV','XSTN','XSVL','XSVT','XSDL','XSDN','XSDT']
        try:
            pool = workerpool.WorkerPool(size=10)
            for code in mienBac:
                pool.put(Task(code, 'Miền Bắc', self.processDetail))
            for code in mienTrung:
                pool.put(Task(code, 'Miền Trung', self.processDetail))
            for code in mienNam:
                pool.put(Task(code, 'Miền Nam', self.processDetail))
            pool.shutdown()
            pool.wait()
        except:
            LOG.error(traceback.format_exc())
        LOG.debug('finished process')
        

if __name__ == '__main__':
    LOG.info('start crawler wapvn_xoso')
    xoso = WapVN(MONGO_SERVER, MONGO_PORT)
#    xoso = WapVN()
    xoso.process()
    LOG.info('finished crawler wapvn_xoso')
    pid = os.getpid()
    os._exit(1)
    os.kill(pid, 9)
